﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Data.OleDb;
using System.Diagnostics;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Runtime.Remoting.Messaging;
using System.Text;
using System.Text.RegularExpressions;
using System.Web;
using System.Windows.Forms;
using Excel;
using ExcelExportLib;
using NCrawler.Extensions;
using NCrawler.HtmlProcessor;
using NCrawler.LanguageDetection.Google;
using NCrawler.Interfaces;
using NCrawler;
using MassPageRankChecker.Properties;
using System.Threading.Tasks;
using MassUtility;
namespace MassPageRankChecker
{
    public partial class MassPageRankChecker : Form
    {
        public CookieContainer sData = new CookieContainer();
        string oldLink = "";
        public MassPageRankChecker()
        {
            InitializeComponent();
        }

        private void btnThucHien_Click(object sender, EventArgs e)
        {
            if (rdWebSite.Checked )
            {
                quetWebSite();
            }
            if (rdTuFileText.Checked )
            {
                quetDataGrid();
            }
            if (rdTuGoogle.Checked  )
            {
                quetGoogle();
            }
           
        }

        private void quetGoogle()
        {
            try
            {
                string thoigian = "";
                if (rd24h.Checked)
                {
                    thoigian = "qdr:d";
                }
                if (rdTuanQua.Checked)
                {
                    thoigian = "qdr:w";
                }
                if (rdThangQua.Checked)
                {
                    thoigian = "qdr:m";
                }
                new System.Action<string, string, string, bool, bool>(beginSearch).BeginInvoke(
                      uxTuKhoa.Text, "", thoigian, chkForum.Checked, chkGoogleVietnam.Checked,
                         new AsyncCallback(endSearch),
                         null);
            }
            catch (Exception)
            {
                    
               // throw;
            }
        }
        private void endSearch(IAsyncResult ar)
        {
            AsyncResult result = (AsyncResult)ar;
            System.Action<string, string, string, bool, bool> action = (System.Action<string, string, string, bool, bool>)result.AsyncDelegate;
            try
            {
                action.EndInvoke(ar);

            }
            catch (Exception Ex)
            {
                
            }
            finally
            {

            }
        }
        private void EndActive(IAsyncResult ar)
        {
            AsyncResult result = (AsyncResult)ar;
            System.Action<string> action = (System.Action<string>)result.AsyncDelegate;
            try
            {

                action.EndInvoke(ar);

            }
            catch (Exception Ex)
            {
                // var del = this.OnError;
                // if (del != null)
                //  {
                //     del(this, new TwitterErrorArgs(Ex));
                // }
            }
            finally
            {

            }
        }
        public void beginSearch(string sSearch, string title, string khoangthoigian, bool bForum, bool bVN)
        {

            string s = getIE();

            int i = 0;
            while (true)
            {
                if (getResult(s, sSearch, title, i, khoangthoigian, bForum, bVN))
                {
                    i++;
                }
                else
                {
                    break;
                }
            }

            // MessageBox.Show("Đã tìm xong");
        }
        public string getResponse(string url)
        {
            try
            {
                StreamReader sr = null;
                HttpWebRequest request = null;
                HttpWebResponse response = null;
                Stream stream;
                StreamReader streamReader;
                request = (HttpWebRequest)WebRequest.Create(url);
                request.AllowAutoRedirect = true;
                request.Accept = "image/jpeg, application/x-ms-application, image/gif, application/xaml+xml,image/pjpeg, application/x-ms-xbap, application/x-shockwave-flash,application/vnd.ms-excel, application/msword, */*";

                request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                request.ServicePoint.ConnectionLimit = 2;
                request.KeepAlive = true;
                // request.Headers.Add("Connection: keep-alive");
                request.Headers.Add("Keep-Alive: 115");
                request.Headers.Add("Accept-Language: en-US");
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ConnectionGroupName = Guid.NewGuid().ToString();
                request.CookieContainer = sData;
                request.AllowWriteStreamBuffering = true;
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                catch (WebException ex)
                {
                    if (ex.Status == WebExceptionStatus.ProtocolError)
                        response = (HttpWebResponse)ex.Response;
                }

                if ((int)response.StatusCode == 404)
                {
                    // MessageBox.Show(response.StatusCode.ToString()  );
                    return "";
                }

                sr = new StreamReader(response.GetResponseStream());

                string loginResponse = sr.ReadToEnd();

                return loginResponse;
            }
            catch (Exception)
            {

                return "";
            }
        }
        public string getIE()
        {
            try
            {


                StreamReader sr = null;
                HttpWebRequest request = null;
                HttpWebResponse response = null;
                Stream stream;
                StreamReader streamReader;
                request = (HttpWebRequest)WebRequest.Create("http://www.google.com.vn/");
                request.AllowAutoRedirect = true;
                request.Accept = "image/jpeg, application/x-ms-application, image/gif, application/xaml+xml,image/pjpeg, application/x-ms-xbap, application/x-shockwave-flash,application/vnd.ms-excel, application/msword, */*";

                request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                request.ServicePoint.ConnectionLimit = 2;
                request.KeepAlive = true;
                // request.Headers.Add("Connection: keep-alive");
                request.Headers.Add("Keep-Alive: 115");
                request.Headers.Add("Accept-Language: en-US");
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ConnectionGroupName = Guid.NewGuid().ToString();
                request.CookieContainer = sData;
                request.AllowWriteStreamBuffering = true;
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                catch (WebException ex)
                {
                    if (ex.Status == WebExceptionStatus.ProtocolError)
                        response = (HttpWebResponse)ex.Response;
                }

                if ((int)response.StatusCode == 404)
                {
                    // MessageBox.Show(response.StatusCode.ToString()  );
                    return "";
                }

                sr = new StreamReader(response.GetResponseStream());

                string loginResponse = sr.ReadToEnd();
                HtmlAgilityPack.HtmlDocument docp = new HtmlAgilityPack.HtmlDocument();
                string s = loginResponse.Substring(loginResponse.IndexOf("ei=\",\"") + 6);
                s = s.Substring(0, s.IndexOf("\""));
                return s;
            }
            catch (Exception)
            {

                return "";
            }
        }
        public bool getResult(string ie, string keyword, string title, int pageNo, string thoigian, bool bForum, bool bVN)
        {
            try
            {


                //"http://www.google.com.vn/webhp?as_q=&hl=vi&num=10&lr=lang_vi&cr=countryVN"
                //http://www.google.com.vn/search?q=cong+hoa+xa+hoi&num=100&hl=vi&lr=&as_qdr=all&prmd=ivns&ei=8NPQTeCROYaGvAP07OyrCg&start=100&sa=N
                StreamReader sr = null;
                HttpWebRequest request = null;
                HttpWebResponse response = null;
                Stream stream;
                StreamReader streamReader;
                string sUrl = "http://www.google.com.vn/search?q={0}&num=100&hl=vi&lr=&as_qdr=all&prmd=ivns&ei={1}&start={2}&sa=N";
                if (thoigian.Length > 0)
                {
                    sUrl = "http://www.google.com.vn/search?q={0}&num=100&hl=vi&lr=lang_vi&tbs=lr:lang_1vi," + thoigian + "&prmd=ivnsfd&&ei={1}&start={2}&sa=N";
                }
                if (!bVN)
                {
                    sUrl = sUrl.Replace("http://www.google.com.vn", "http://www.google.com");
                    sUrl = sUrl.Replace("vi", "us");
                }
                if (bForum)
                {
                    keyword = keyword + " \"powered by vbulletin\" inurl:/forum OR inurl:/community OR inurl:/diendan";
                }
                sUrl = string.Format(sUrl, keyword, ie, (pageNo * 100).ToString());
                request = (HttpWebRequest)WebRequest.Create(sUrl);
                request.AllowAutoRedirect = true;
                request.Accept = "image/jpeg, application/x-ms-application, image/gif, application/xaml+xml,image/pjpeg, application/x-ms-xbap, application/x-shockwave-flash,application/vnd.ms-excel, application/msword, */*";

                request.AutomaticDecompression = DecompressionMethods.Deflate | DecompressionMethods.GZip;
                request.ServicePoint.ConnectionLimit = 2;
                request.KeepAlive = true;
                // request.Headers.Add("Connection: keep-alive");
                request.Headers.Add("Keep-Alive: 115");
                request.Headers.Add("Accept-Language: en-US");
                request.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 6.1; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0)";
                request.ContentType = "application/x-www-form-urlencoded";
                request.ConnectionGroupName = Guid.NewGuid().ToString();
                request.CookieContainer = sData;
                request.AllowWriteStreamBuffering = true;
                try
                {
                    response = (HttpWebResponse)request.GetResponse();
                }
                catch (WebException ex)
                {
                    if (ex.Status == WebExceptionStatus.ProtocolError)
                        response = (HttpWebResponse)ex.Response;
                }

                if ((int)response.StatusCode == 404)
                {
                    // MessageBox.Show(response.StatusCode.ToString()  );
                    return false;
                }

                sr = new StreamReader(response.GetResponseStream());

                string loginResponse = sr.ReadToEnd();
                DumpHRefs(loginResponse, title);
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(loginResponse);
                var pn = doc.DocumentNode.SelectNodes("//a[@id='pnnext']");
                if (pn != null)
                {
                    return true;
                }
                else
                    return false;


            }
            catch (Exception)
            {

                return true;
            }

        }
        private bool DumpHRefs(string inputString, string title)
        {
            try
            {


                Match m;
                bool haveLink = false;
                string HRefPattern = "href\\s*=\\s*(?:\"(?<1>[^\"]*)\"|(?<1>\\S+))";
                HtmlAgilityPack.HtmlDocument doc = new HtmlAgilityPack.HtmlDocument();
                doc.LoadHtml(inputString);
                var li = doc.DocumentNode.SelectNodes("//div[@class='vsc']");
                //title = StringUtil.RemoveSign4VietnameseString(title).ToLower();
                string[] skey = title.Split(',');
                foreach (var item in li)
                {
                    try
                    {
                        haveLink = true;
                        HtmlAgilityPack.HtmlDocument doc1 = new HtmlAgilityPack.HtmlDocument();
                        doc1.LoadHtml(item.OuterHtml);
                        var vs = doc1.DocumentNode.SelectNodes("//div[@class='s']").FirstOrDefault();
                        var vt = doc1.DocumentNode.SelectNodes("//a[@class='l']").FirstOrDefault();
                        string link = HttpUtility.HtmlDecode(vt.Attributes["href"].Value.ToString());
                        int iPageRank = GooglePageRank.GetPageRank(link);
                        int iAlexa = AlexaCheckRank.getAlexa(link);
                        string sc = getResponse(link);
                        string rel = "";
                        if (sc.Contains("nofollow"))
                        {
                            rel = rel + ",nofollow";

                        }
                        else
                        if (sc.Contains("dofollow"))
                        {
                            rel = rel + ",dofollow";

                        }
                        else
                        {
                            if (sc.Contains("follow"))
                            {
                                rel = rel + ",follow";

                            }   
                        }
                        if (rel.StartsWith(","))
                        {
                            rel = rel.Substring(1);
                        }
                        Invoke(new MethodInvoker(delegate()
                        {
                            dataGridView1.Rows.Add(dataGridView1.Rows.Count, link, iPageRank, iAlexa, rel);
                        }));
                        
                    }
                    catch (Exception ex)
                    {
                        MessageBox.Show(ex.ToString());
                        //   throw;
                    }

                }

                return haveLink;
            }
            catch (Exception)
            {
                return true;
                //   throw;
            }
        }
        private void quetDataGrid()
        {
            try
            {
                var bw = new BackgroundWorker();
  bw.WorkerReportsProgress = true;
  bw.DoWork += delegate {
    // do work. You can use locals from here
      for (int i = 0; i < dataGridView1.RowCount ; i++)
      {
          try
          {

          
          string url = dataGridView1.Rows[i].Cells[1].Value.ToString();
          int iPageRank = GooglePageRank.GetPageRank(url);
          int iAlexa = AlexaCheckRank.getAlexa(url);
          string sc = getResponse(url);
          string rel = "";
          if (sc.Contains("nofollow"))
          {
              rel = rel + ",nofollow";

          }
          else
              if (sc.Contains("dofollow"))
              {
                  rel = rel + ",dofollow";

              }
              else
              {
                  if (sc.Contains("follow"))
                  {
                      rel = rel + ",follow";

                  }
              }
          if (rel.StartsWith(","))
          {
              rel = rel.Substring(1);
          }
          dataGridView1.Rows[i].Cells[2].Value=iPageRank ;
          dataGridView1.Rows[i].Cells[3].Value = iAlexa;
          dataGridView1.Rows[i].Cells[4].Value = rel;
          }
          catch (Exception)
          {

           //   throw;
          }
      }
  };
  //bw.ProgressChanged += delegate { ... };
  bw.RunWorkerCompleted += delegate {
    // do something with the results.
                                        
  };
  bw.RunWorkerAsync();
            }
            catch (Exception)
            {
                    
                //throw;
            }
        }

        private void quetWebSite()
        {
            try
            {
                NCrawlerModule.Setup();
                DumperStep ds = new DumperStep();
                ds.OnLinkCrawled += new DumperStep.ResponseLinkCrawled(ds_OnLinkCrawled);
                Task.Factory.StartNew(() =>
                {
                    using (Crawler c = new Crawler(new Uri(uxWebsite.Text.Trim()),
                        new HtmlDocumentProcessor(), // Process html
                        ds)
                    {
                        // Custom step to visualize crawl
                        MaximumThreadCount = Convert.ToInt32(uxSoLuongChay.Value),
                        MaximumCrawlDepth = Convert.ToInt32(uxDoSau.Value),
                        ExcludeFilter = Program.ExtensionsToSkip,

                    })
                    {
                        c.Crawl();

                    }
                }); 
            }
            catch (Exception)
            {
                    
               // throw;
            }
        }

        void ds_OnLinkCrawled(object sender, string e,string rel)
        {
            try
            {

            
            int iPageRank=0;
            int iAlexa=0;
            Task.Factory.StartNew(() => {
                iPageRank = GooglePageRank.GetPageRank(e);
                iAlexa = AlexaCheckRank.getAlexa(e);
                Invoke(new MethodInvoker(delegate()
               {
                   dataGridView1.Rows.Add(dataGridView1.RowCount + 1, e, iPageRank, iAlexa, rel);  
                    })); 
            });
            }
            catch (Exception)
            {

             //   throw;
            }    
           
            
          //  Console.WriteLine("Url: {0}", e);
        }

        private void MassPageRankChecker_Load(object sender, EventArgs e)
        {
            //Task.Factory.StartNew(() => {
            webBrowser1.Navigate("http://www.youtube.com/watch?v=AI0K28CBsDQ");  
          //  }); 
        }

        private void linkLabel1_LinkClicked(object sender, LinkLabelLinkClickedEventArgs e)
        {
            try
            {


                System.Diagnostics.ProcessStartInfo
    procFormsBuilderStartInfo = new System.Diagnostics.ProcessStartInfo();
                procFormsBuilderStartInfo.FileName = linkLabel1.Text;

                procFormsBuilderStartInfo.WindowStyle =
                System.Diagnostics.ProcessWindowStyle.Maximized;
                System.Diagnostics.Process procFormsBuilder = new
                System.Diagnostics.Process();
                procFormsBuilder.StartInfo = procFormsBuilderStartInfo;
                procFormsBuilder.Start();
            }
            catch (Exception)
            {

                //   throw;
            }
        }

        private void btnMoFile_Click(object sender, EventArgs e)
        {
            openFileDialog1.InitialDirectory = "c:\\";
            openFileDialog1.Filter = "Text files (*.txt)|*.txt|Access files (*.mdb)|*.mdb|Excel files (*.xls)|*.xls|Excel files (*.xlsx)|*.xlsx";
            openFileDialog1.FilterIndex = 2;
            openFileDialog1.RestoreDirectory = true;

            if (openFileDialog1.ShowDialog() == DialogResult.OK)
            {
                uxTextFile.Text = openFileDialog1.FileName;
            }
            if (uxTextFile.Text.Length < 10)
            {
                return;
            }
            if (uxTextFile.Text.EndsWith(".txt"))
            {
                try
                {
                    // Create an instance of StreamReader to read from a file.
                    // The using statement also closes the StreamReader.
                    using (StreamReader sr = new StreamReader(uxTextFile.Text))
                    {
                        string line;
                        // Read and display lines from the file until the end of
                        // the file is reached.
                        while ((line = sr.ReadLine()) != null)
                        {
                            // Console.WriteLine(line);
                            if (line.Trim().Length > 12)
                            {
                                dataGridView1.Rows.Add(dataGridView1.RowCount + 1, true, line);
                            }

                        }
                    }
                }
                catch (Exception ex)
                {

                }
            }
            if (uxTextFile.Text.EndsWith(".xlsx") || uxTextFile.Text.EndsWith(".xls"))
            {
                FileStream stream = File.Open(uxTextFile.Text, FileMode.Open, FileAccess.Read);
                int iCol = 0;
                //1. Reading from a binary Excel file ('97-2003 format; *.xls)
                if (uxTextFile.Text.EndsWith(".xls"))
                {
                    IExcelDataReader excelReader = ExcelReaderFactory.CreateBinaryReader(stream);
                    excelReader.IsFirstRowAsColumnNames = true;
                    DataSet result = excelReader.AsDataSet();

                    //5. Data Reader methods
                    foreach (DataRow sr in result.Tables[0].Rows)
                    {
                        string ss = sr.ItemArray[0].ToString();
                        if (ss.Length > 12)
                        {
                            dataGridView1.Rows.Add(dataGridView1.RowCount + 1,  ss);
                        }
                    }

                    //6. Free resources (IExcelDataReader is IDisposable)
                    excelReader.Close();
                }
                if (uxTextFile.Text.EndsWith(".xlsx"))
                {
                    IExcelDataReader excelReader = ExcelReaderFactory.CreateOpenXmlReader(stream);
                    excelReader.IsFirstRowAsColumnNames = true;
                    DataSet result = excelReader.AsDataSet();

                    //5. Data Reader methods
                    foreach (DataRow sr in result.Tables[0].Rows)
                    {
                        string ss = sr.ItemArray[0].ToString();
                        if (ss.Length > 12)
                        {
                            dataGridView1.Rows.Add(dataGridView1.RowCount + 1,  ss);
                        }
                    }

                    excelReader.Close();
                }



            }
        }

        private void btnXuatExcel_Click(object sender, EventArgs e)
        {
            try
            {
                var sFile = Application.StartupPath + "\\CheckPageRankAlexa" + DateTime.Now.ToString("ddMMyyHHmmss") + ".xls";
                ExcelDocument document = new ExcelDocument();
                document.Company = "MassForumPoster";
                document.Author = "MassForumPoster";
                document.LastAuthor = "MassForumPoster";
                document.Description = "Kết quả" + Environment.NewLine + " Kiểm tra PR";
                document.Title = "LIST PAGERANK - ALEXA RANK";
                ExcelTable table = document.AddTable("PR-Alexa Rank");
                ExcelCell ec;

                table.AddColumn(true, 50);
                table.AddColumn(true, 450);
                table.AddColumn(true, 70);
                table.AddColumn(true, 70);
                ExcelRow row;
                for (int i = 0; i < dataGridView1.RowCount; i++)
                {
                    row = table.AddRow();
                    row.AddCell(i.ToString());
                    row.AddCell(dataGridView1.Rows[i].Cells[1].Value.ToString());
                    row.AddCell(dataGridView1.Rows[i].Cells[2].Value.ToString());
                    row.AddCell(dataGridView1.Rows[i].Cells[3].Value.ToString());
                }

                document.Save(sFile);
                RunExcel1(sFile);

                // System.Diagnostics.Process.Start(sFile);

            }
            catch (Exception ex)
            {

                MessageBox.Show(ex.ToString());
            }
        }
        private static void RunExcel1(string filename)
        {
            try
            {
                System.Diagnostics.Process.Start("Excel", string.Concat("\"", filename, "\""));
            }
            catch (Exception ex)
            {
                RunExcel(filename);
            }
        }

        private static void RunExcel(string filename)
        {
            ProcessStartInfo startInfo = new ProcessStartInfo(filename);
            startInfo.UseShellExecute = true;
            Process.Start(startInfo);
        }

       
    }
    public class DumperStep : IPipelineStep
    {
        public event ResponseLinkCrawled OnLinkCrawled;
        public delegate void ResponseLinkCrawled(object sender, string e,string rel);
        #region IPipelineStep Members

        /// <summary>
        /// </summary>
        /// <param name="crawler">
        /// The crawler.
        /// </param>
        /// <param name="propertyBag">
        /// The property bag.
        /// </param>
        public void Process(Crawler crawler, PropertyBag propertyBag)
        {


            lock (this)
            {
               // Settings.Default.LinkCrawler1 = propertyBag.Step.Uri.ToString();
              //  Settings.Default.Save();
                var sr = new StreamReader(propertyBag.GetResponse());
               string sc= sr.ReadToEnd();
               string rel = "";
               if (sc.Contains("nofollow"))
               {
                   rel = rel + ",nofollow";

               }
               else
                   if (sc.Contains("dofollow"))
                   {
                       rel = rel + ",dofollow";

                   }
                   else
                   {
                       if (sc.Contains("follow"))
                       {
                           rel = rel + ",follow";

                       }
                   }
                if (rel.StartsWith(","))
                {
                    rel = rel.Substring(1);
                }
                OnLinkCrawled(this, propertyBag.Step.Uri.ToString(), rel);
               
            }
        }

        #endregion
    }
}
